# -*- coding: utf-8 -*-
import scrapy

import re
import urllib.parse
import logging
from pprint import pprint
from copy import deepcopy, copy
from FDC_spider.items import FdcEstateGuidItem, FdcBuildingItem, FdcRoomItem
from FDC_spider.constants import STATUS_DICT

logger = logging.getLogger(__name__)


class BeijingSpider(scrapy.Spider):
    name = 'beijing'
    allowed_domains = ['beijing.gov.cn']
    start_urls = ['http://zjw.beijing.gov.cn/bjjs/fwgl/fdcjy/index.shtml']
    # estate_url = 'http://zjw.beijing.gov.cn/eportal/ui?pageId=307670'   # 2020/4/26 维护升级
    estate_url = 'http://bjjs.zjw.beijing.gov.cn/eportal/ui?pageId=307670'  # POST    楼盘url

    def parse(self, response):
        # estate_li_url_temp = 'http://zjw.beijing.gov.cn/eportal/ui?pageId=307670&rblFWType={}&txtYS=&txtZH='  # 2020/4/26 维护升级
        estate_li_url_temp = 'http://bjjs.zjw.beijing.gov.cn/eportal/ui?pageId=307670&rblFWType={}&txtYS=&txtZH='
        # 期房/现房
        estate_type_li = response.xpath("//input[@name='xinjianspf']")
        # for estate_type in estate_type_li[1:2]:
        for estate_type in estate_type_li:
            item_eg = FdcEstateGuidItem()
            item_eg['isCompletionAcceptance'] = estate_type.xpath("./@title").extract_first()
            item_eg['estateTypeParams'] = estate_type.xpath("./@value").extract_first()
            yield scrapy.Request(
                estate_li_url_temp.format(item_eg['estateTypeParams']),
                callback=self.parse_estate_li,
                meta={'item_eg': deepcopy(item_eg)},
            )

    def parse_estate_li(self, response):
        """
        获取楼盘列表信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        # 获取项目分组
        tr_li = response.xpath("//th[text()='项目名称']/../following-sibling::tr")
        if len(tr_li) > 0:
            for tr in tr_li:
                projectName = tr.xpath("./td[1]//text()").extract_first()
                item_eg['projectName'] = projectName if projectName else None
                preSalePermit = tr.xpath("./td[2]//text()").extract_first()
                item_eg['preSalePermit'] = preSalePermit if preSalePermit else None
                certDate = tr.xpath("./td[3]//text()").extract_first()
                item_eg['certDate'] = certDate if certDate else None
                item_eg['estateUrl'] = tr.xpath("./td[1]//@href").extract_first()
                yield response.follow(
                    item_eg['estateUrl'],
                    callback=self.parse_estate,
                    meta={'item_eg': deepcopy(item_eg)},
                )

        # 翻页
        info_li = response.xpath("//td[@class='Normal']/text()").extract()
        params_li = re.findall(r'(\d+)', ''.join(info_li))
        total_num = params_li[0]
        page_size = params_li[1]
        current_page_num = response.xpath("//input[@id='jumpPageBox']/@value").extract_first()
        total_page_num = int(total_num) // int(page_size) if int(total_num) % int(page_size) == 0 else int(
            total_num) // int(page_size) + 1
        if int(current_page_num) < int(total_page_num):
            # 构造下一页请求参数
            next_data_dict = dict(
                projectName='项目名称关键字',
                rblFWType=item_eg['estateTypeParams'],
                txtYS='',
                txtZH='',
                txtCQZH='证号关键字',
                developer='单位名称关键字',
                txtaddress='地址关键字',
                dopIsHouse='-1',
                currentPage=str(int(current_page_num) + 1),
                pageSize=str(params_li[1]),
            )
            yield scrapy.FormRequest(
                BeijingSpider.estate_url,
                callback=self.parse_estate_li,
                formdata=next_data_dict,
                meta={'item_eg': deepcopy(item_eg)},
            )

    def parse_estate(self, response):
        """
        获取楼盘信息
        :param response:
        :return:
        """
        item_eg = copy(response.meta['item_eg'])
        item_eg['estateUrl'] = response.request.url
        qs_dict = urllib.parse.parse_qs(item_eg['estateUrl'].split('?')[1])
        item_eg['projectId'] = qs_dict['projectID'][0]
        if int(qs_dict['systemID'][0]) == 3:
            estateAddress = response.xpath("//td[@id='坐落位置']/text()").extract_first()
            item_eg['estateAddress'] = estateAddress.strip() if estateAddress else None
            try:
                item_eg['districtName'] = item_eg['estateAddress'].split('区')[0] + '区'
            except Exception as e:
                logger.error('{}:获取行政区域失败  error_msg:{}'.format(item_eg['projectName'], e))
                item_eg['districtName'] = None
            enterpriseName = response.xpath("//td[@id='房屋所有权人名称']/text()").extract_first()
            item_eg['enterpriseName'] = enterpriseName.strip() if enterpriseName else None
            registrationDepartment = response.xpath("//td[@id='转移登记办理部门']/text()").extract_first()
            item_eg['registrationDepartment'] = registrationDepartment.strip() if registrationDepartment else None
            landUseRightNum = response.xpath("//td[@id='土地使用权证号']/text()").extract_first()
            item_eg['landUseRightNum'] = landUseRightNum.strip() if landUseRightNum else None
            yield item_eg

        else:
            estateAddress = response.xpath("//td[@id='坐落位置']/text()").extract_first()
            item_eg['estateAddress'] = estateAddress.strip() if estateAddress else None
            try:
                item_eg['districtName'] = item_eg['estateAddress'].split('区')[0] + '区'
            except Exception as e:
                logger.error('{}:获取行政区域失败  error_msg:{}'.format(item_eg['projectName'], e))
                item_eg['districtName'] = None
            enterpriseName = response.xpath("//td[@id='开发企业']/text()").extract_first()
            item_eg['enterpriseName'] = enterpriseName.strip() if enterpriseName else None
            registrationDepartment = response.xpath("//td[@id='预售登记备案管理部门']/text()").extract_first()
            item_eg['registrationDepartment'] = registrationDepartment.strip() if registrationDepartment else None
            CA = response.xpath("//td[@id='许可机关']/text()").extract_first()
            item_eg['CA'] = CA.strip() if CA else None
            landUsageTerm = response.xpath("//td[@id='土地用途和使用年限']/text()").extract_first()
            item_eg['landUsageTerm'] = landUsageTerm.strip() if landUsageTerm else None
            approvalArea = response.xpath("//td[@id='准许销售面积']/text()").extract_first()
            item_eg['approvalArea'] = approvalArea.strip() if approvalArea else None
            approvalParts = response.xpath("//td[@id='批准预售部位']/text()").extract_first()
            item_eg['approvalParts'] = approvalParts.strip() if approvalParts else None
            preSaleDepositBank = response.xpath("//td[@id='预售资金监管银行']/text()").extract_first()
            item_eg['preSaleDepositBank'] = preSaleDepositBank.strip() if preSaleDepositBank else None
            preSaleAccountName = response.xpath("//td[@id='专用账户名称']/text()").extract_first()
            item_eg['preSaleAccountName'] = preSaleAccountName.strip() if preSaleAccountName else None
            preSaleAccountNum = response.xpath("//td[@id='专用账户账号']/text()").extract_first()
            item_eg['preSaleAccountNum'] = preSaleAccountNum.strip() if preSaleAccountNum else None
            yield item_eg

        item_bd = FdcBuildingItem()
        item_bd['isCompletionAcceptance'] = item_eg['isCompletionAcceptance']
        item_bd['projectName'] = item_eg['projectName']
        item_bd['districtName'] = item_eg['districtName']
        item_bd['estateAddress'] = item_eg['estateAddress']
        item_bd['enterpriseName'] = item_eg['enterpriseName']
        # 楼栋列表url
        building_li_url = response.xpath("//a[contains(text(),'查看更多')]/@href").extract_first()
        if building_li_url:
            yield response.follow(
                building_li_url,
                callback=self.parse_building_li,
                meta={'item_bd': deepcopy(item_bd)}
            )
        else:
            # 获取楼栋列表分组
            tr_li = response.xpath("//strong[text()='楼盘表']/../following-sibling::tr")
            if len(tr_li.extract()) > 0:
                for tr in tr_li:
                    item_bd['blockName'] = tr.xpath("./td[1]/text()").extract_first()
                    item_bd['approvalRoomNo'] = tr.xpath("./td[2]/text()").extract_first()
                    item_bd['approvalArea'] = tr.xpath("./td[3]/text()").extract_first()
                    item_bd['saleStatus'] = tr.xpath("./td[4]/text()").extract_first()
                    # 楼栋url
                    building_url = tr.xpath("./td[6]/a/@href").extract_first()
                    building_url = building_url.strip() if building_url else None
                    if building_url and building_url != '#':
                        yield response.follow(
                            building_url,
                            callback=self.parse_building,
                            meta={'item_bd': deepcopy(item_bd)}
                        )
            else:
                logger.warning('{}：无法获取楼栋信息'.format(item_bd['projectName']))

    def parse_building_li(self, response):
        """
        获取楼栋列表信息
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        qs_dict = urllib.parse.parse_qs(response.request.url.split('?')[1])
        qs_dict['systemId'] = qs_dict['systemId'][0]
        # 获取当前页楼栋列表分组
        tr_li = response.xpath("//th[text()='楼盘表']/../following-sibling::tr")
        if len(tr_li) > 0:
            for tr in tr_li:
                item_bd['blockName'] = tr.xpath("./td[1]/a/text()").extract_first()
                if int(qs_dict['systemId']) == 3:
                    if len(tr.xpath("./td").extract()) == 5:
                        item_bd['approvalRoomNo'] = tr.xpath("./td[2]/text()").extract_first()
                        item_bd['approvalArea'] = tr.xpath("./td[3]/text()").extract_first()
                        item_bd['saleStatus'] = tr.xpath("./td[4]/text()").extract_first()
                        # 楼栋url
                        building_url = tr.xpath("./td[5]/a/@href").extract_first()
                        building_url = building_url.strip() if building_url else None
                        if building_url and building_url != '#':
                            yield response.follow(
                                building_url,
                                callback=self.parse_building,
                                meta={'item_bd': deepcopy(item_bd)}
                            )
                        else:
                            item_bd['floorOverGroundNo'] = None
                            item_bd['floorUnderGroundNo'] = None
                            item_bd['floorTotalNo'] = None
                            item_bd['buildingTotalArea'] = None
                            yield item_bd
                    else:
                        item_bd['approvalRoomNo'] = None
                        item_bd['approvalArea'] = None
                        item_bd['saleStatus'] = '已售完'
                        item_bd['floorOverGroundNo'] = None
                        item_bd['floorUnderGroundNo'] = None
                        item_bd['floorTotalNo'] = None
                        item_bd['buildingTotalArea'] = None
                        yield item_bd

                else:
                    item_bd['approvalRoomNo'] = tr.xpath("./td[2]/text()").extract_first()
                    item_bd['approvalArea'] = tr.xpath("./td[3]/text()").extract_first()
                    item_bd['saleStatus'] = tr.xpath("./td[4]/text()").extract_first()
                    # 楼栋url
                    if len(tr.xpath("./td").extract()) == 6:
                        building_url = tr.xpath("./td[6]/a/@href").extract_first()
                        building_url = building_url.strip() if building_url else None
                    else:
                        building_url = tr.xpath("./td[5]/a/@href").extract_first()
                        building_url = building_url.strip() if building_url else None
                    if building_url and building_url != '#':
                        yield response.follow(
                            building_url,
                            callback=self.parse_building,
                            meta={'item_bd': deepcopy(item_bd)}
                        )
                    else:
                        item_bd['floorOverGroundNo'] = None
                        item_bd['floorUnderGroundNo'] = None
                        item_bd['floorTotalNo'] = None
                        item_bd['buildingTotalArea'] = None
                        yield item_bd

        # 翻页
        info_li = response.xpath("//td[@class='Normal']/text()").extract()
        params_li = re.findall(r'(\d+)', ''.join(info_li))
        total_num = params_li[0]
        page_size = params_li[1]
        current_page_num = response.xpath("//input[@id='jumpPageBox']/@value").extract_first()
        total_page_num = int(total_num) // int(page_size) if int(total_num) % int(page_size) == 0 else int(
            total_num) // int(page_size) + 1
        if int(current_page_num) < int(total_page_num):
            # 构造下一页请求参数
            next_data_dict = dict(
                currentPage=str(int(current_page_num) + 1),
                pageSize=str(page_size),
            )
            yield scrapy.FormRequest(
                response.request.url,
                callback=self.parse_building_li,
                formdata=next_data_dict,
                meta={'item_bd': deepcopy(item_bd)},
            )

    def parse_building(self, response):
        """
        获取楼栋信息
        :param response:
        :return:
        """
        item_bd = copy(response.meta['item_bd'])
        try:
            buildingTotalArea_str = response.xpath("//font[contains(text(),'地上')]/../font[1]/text()").extract_first()
            buildingTotalArea_li = re.findall(r'建筑面积:(.*)m', buildingTotalArea_str)
            item_bd['buildingTotalArea'] = buildingTotalArea_li[0] if len(buildingTotalArea_li) > 0 else None
        except Exception as e:
            logger.warning('获取{}-{}建筑面积失败：{}'.format(item_bd['projectName'], item_bd['blockName'], e))
            item_bd['buildingTotalArea'] = None
        try:
            floorOverGroundNo_str = response.xpath("//font[contains(text(),'地上')]/../font[2]/text()").extract_first()
            floorOverGroundNo_li = re.findall(r'地上:(.*)层', floorOverGroundNo_str)
            item_bd['floorOverGroundNo'] = floorOverGroundNo_li[0] if len(floorOverGroundNo_li) > 0 else 0
        except Exception as e:
            logger.warning('获取{}-{}地上层数失败：{}'.format(item_bd['projectName'], item_bd['blockName'], e))
            item_bd['floorOverGroundNo'] = 0
        try:
            floorUnderGroundNo_str = response.xpath("//font[contains(text(),'地上')]/../font[3]/text()").extract_first()
            floorUnderGroundNo_li = re.findall(r'地下:(.*)层', floorUnderGroundNo_str)
            item_bd['floorUnderGroundNo'] = floorUnderGroundNo_li[0] if len(floorUnderGroundNo_li) > 0 else 0
        except Exception as e:
            logger.warning('获取{}-{}地下层数失败：{}'.format(item_bd['projectName'], item_bd['blockName'], e))
            item_bd['floorUnderGroundNo'] = 0
        try:
            item_bd['floorTotalNo'] = str(int(item_bd['floorOverGroundNo']) + int(item_bd['floorUnderGroundNo']))
        except Exception as e:
            logger.warning('获取{}-{}总楼层失败：{}'.format(item_bd['projectName'], item_bd['blockName'], e))
            item_bd['floorTotalNo'] = 0
        else:
            # 获取当前楼栋的楼层分组
            tr_li = response.xpath("//strong[contains(text(),'号')]/../../following-sibling::tr")
            if len(tr_li) > 0:
                for tr in tr_li:
                    # 获取当前楼层的房号分组
                    div_li = tr.xpath("./td[3]/div")
                    if len(div_li.extract()) > 0:
                        for div in div_li:
                            item_rm = FdcRoomItem()
                            item_rm['projectName'] = item_bd['projectName']
                            item_rm['isCompletionAcceptance'] = item_bd['isCompletionAcceptance']
                            item_rm['districtName'] = item_bd['districtName']
                            item_rm['enterpriseName'] = item_bd['enterpriseName']
                            item_rm['blockName'] = item_bd['blockName']
                            item_rm['naturalFloor'] = tr.xpath("./td[1]/text()").extract_first()
                            item_rm['salesFloor'] = tr.xpath("./td[2]/text()").extract_first()
                            item_rm['roomNo'] = div.xpath("./a/text()").extract_first()
                            style_str = div.xpath("./@style").extract_first()
                            try:
                                status_key = re.findall(r'background:(.*);', style_str)[0].split(';')[0]
                                item_rm['saleStatus'] = STATUS_DICT.get(status_key, None)
                            except Exception as e:
                                logger.error('{}_{}_{}：获取房号销售状态失败   error_msg:{}'.format(item_bd['projectName'],
                                                                                         item_rm['blockName'],
                                                                                         item_rm['roomNo'], e))
                                item_rm['saleStatus'] = None
                            # 构造价格字典
                            qs_dict = urllib.parse.parse_qs(response.request.url.split('?')[1])
                            qs_dict['systemId'] = qs_dict['systemId'][0]
                            if int(qs_dict['systemId']) == 3:
                                price_str = response.xpath(
                                    "//font[contains(text(),'地上')]/../font[4]/text()").extract_first()
                            else:
                                price_str = response.xpath("//font[contains(text(),'拟售')]/text()").extract_first()
                            price_str = re.sub(r'\uff1a', ':', price_str).split(':', maxsplit=1)[1]
                            price_str = price_str.strip() if price_str else None
                            if price_str:
                                price_li = re.sub(r'\s+|&nbsp|;', '_', price_str)
                                price_li = re.sub(r'_{1,}', '_', price_li).split('_')
                                item_rm['priceDict'] = {i.split(':')[0]: i.split(':')[1] for i in price_li if
                                                        i.strip() != ''}
                            else:
                                item_rm['priceDict'] = dict()

                            # 房号url
                            room_url = div.xpath("./a/@href").extract_first()
                            room_url = room_url.strip() if room_url else None
                            if room_url and room_url != '#':
                                yield response.follow(
                                    room_url,
                                    callback=self.parse_room,
                                    meta={'item_rm': deepcopy(item_rm)}
                                )
                            else:
                                item_rm['unitPrice'] = None
                                item_rm['buildingAreaUnitPrice'] = None
                                item_rm['innerAreaUnitPrice'] = None
                                item_rm['roomUse'] = None
                                item_rm['roomStructure'] = None
                                item_rm['buildingArea'] = None
                                item_rm['innerArea'] = None
                                yield item_rm
            else:
                logger.warning('无法获取{}-{}房号信息'.format(item_bd['projectName'], item_bd['blockName']))
            yield item_bd

    def parse_room(self, response):
        """
        获取房号信息
        :param response:
        :return:
        """
        item_rm = copy(response.meta['item_rm'])
        tr_li = response.xpath("//td[contains(text(),'房屋资料')]/../following-sibling::tr").extract()
        if len(tr_li) >= 2:
            roomUse = response.xpath("//td[contains(text(),'途')]/following-sibling::td[1]/text()").extract_first()
            item_rm['roomUse'] = roomUse.strip() if roomUse else None
            item_rm['unitPrice'] = item_rm['priceDict'].get(item_rm['roomUse'], None)

            roomStructure = response.xpath("//td[contains(text(),'型')]/following-sibling::td[1]/text()").extract_first()
            item_rm['roomStructure'] = roomStructure.strip() if roomStructure else None

            buildingArea = response.xpath(
                "//td[contains(text(),'建筑面积')][1]/following-sibling::td[1]/text()").extract_first()
            if buildingArea:
                buildingArea_str = re.sub(r'\s+', '', buildingArea)
                buildingArea_li = re.findall(r'(.*)平', buildingArea_str)
                item_rm['buildingArea'] = buildingArea_li[0] if len(buildingArea_li) > 0 else None
            else:
                item_rm['buildingArea'] = None

            innerArea = response.xpath(
                "//td[contains(text(),'套内面积')][1]/following-sibling::td[1]/text()").extract_first()
            if innerArea:
                innerArea_str = re.sub(r'\s+', '', innerArea)
                innerArea_li = re.findall(r'(.*)平', innerArea_str)
                item_rm['innerArea'] = innerArea_li[0] if len(innerArea_li) > 0 else None
            else:
                item_rm['innerArea'] = None

            buildingAreaUnitPrice = response.xpath(
                "//td[contains(text(),'建筑面积拟售单价')]/following-sibling::td[1]/text()").extract_first()
            if buildingAreaUnitPrice:
                buildingAreaUnitPrice_str = re.sub(r'\s+', '', buildingAreaUnitPrice)
                buildingAreaUnitPrice_li = re.findall(r'(.*)元', buildingAreaUnitPrice_str)
                item_rm['buildingAreaUnitPrice'] = buildingAreaUnitPrice_li[0] if len(
                    buildingAreaUnitPrice_li) > 0 else None
            else:
                item_rm['buildingAreaUnitPrice'] = None

            innerAreaUnitPrice = response.xpath(
                "//td[contains(text(),'套内面积拟售单价')]/following-sibling::td[1]/text()").extract_first()
            if innerAreaUnitPrice:
                innerAreaUnitPrice_str = re.sub(r'\s+', '', innerAreaUnitPrice)
                innerAreaUnitPrice_li = re.findall(r'(.*)元', innerAreaUnitPrice_str)
                item_rm['innerAreaUnitPrice'] = innerAreaUnitPrice_li[0] if len(innerAreaUnitPrice_li) > 0 else None
            else:
                item_rm['innerAreaUnitPrice'] = None

        else:
            item_rm['unitPrice'] = None
            item_rm['buildingAreaUnitPrice'] = None
            item_rm['innerAreaUnitPrice'] = None
            item_rm['roomUse'] = None
            item_rm['roomStructure'] = None
            item_rm['buildingArea'] = None
            item_rm['innerArea'] = None

        yield item_rm
